{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# forward model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'model_weight' : '../ner_model_weight/model_conv_625.h5',\n",
    "    'embed_size' : 500,\n",
    "    'max_sent_len': 20\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('../char_6.17.json', mode='r', encoding='utf-8') as f:\n",
    "    dicts = json.load(f)\n",
    "char2id = dicts['char2id']\n",
    "id2char = dicts['id2char']\n",
    "intent2id = dicts['intent2id']\n",
    "id2intent = dicts['id2intent']\n",
    "slot2id = dicts['slot2id']\n",
    "id2slot = dicts['id2slot']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    y = 1 / (1 + np.exp(-x))\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tanh(x):\n",
    "    y=(np.exp(x)-np.exp(-x))/(np.exp(x)+np.exp(-x))\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embedding(x,embed_size,embed):\n",
    "    x_one= np.zeros((len(x),embed_size))\n",
    "    x_one[range(len(x)), x] = 1\n",
    "    x_embed = np.dot(x_one, embed)\n",
    "    return x_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GRU(cell_inputs,cell_states,kernel,recurrent_kernel,bias):\n",
    "    input_bias = bias[0]\n",
    "    recurrent_bias = bias[1]\n",
    "    \n",
    "    matrix_x = np.dot(cell_inputs, kernel)\n",
    "    matrix_x = np.add(matrix_x, input_bias)\n",
    "    \n",
    "    x_z, x_r, x_h = np.split(matrix_x,3, axis=-1)\n",
    "    \n",
    "    matrix_inner = np.dot(cell_states, recurrent_kernel)\n",
    "    matrix_inner = np.add(matrix_inner, recurrent_bias)\n",
    "    \n",
    "    recurrent_z, recurrent_r, recurrent_h = np.split(matrix_inner,3,axis=-1)\n",
    "    \n",
    "    z = sigmoid(x_z + recurrent_z)\n",
    "    r = sigmoid(x_r + recurrent_r)\n",
    "    hh = tanh(x_h + r * recurrent_h)\n",
    "    \n",
    "    h = z * cell_states + (1-z) * hh\n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GlobalAveragePooling1D(x,step_axis=0):\n",
    "    return np.mean(x,axis=step_axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LayerNormalization(x,gamma,beta,step_axis = -1,epsilon=1e-3):\n",
    "    mean = np.mean(x,axis = step_axis)\n",
    "    mean = np.expand_dims(mean,axis=1)\n",
    "    variance = np.var(x,axis = step_axis)\n",
    "    variance = np.expand_dims(variance,axis=1)\n",
    "    inv = 1.0 / np.sqrt(variance + epsilon)\n",
    "#     print(np.shape(inv))\n",
    "    gamma = np.expand_dims(gamma,axis=0)\n",
    "    beta = np.expand_dims(beta,axis=0)\n",
    "    inv = gamma *inv\n",
    "    return x * inv + (beta - mean * inv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dense(x,gamma, bias):\n",
    "#     print(np.shape(x))\n",
    "    y = np.matmul(x,gamma)\n",
    "    y = np.add(y,bias)\n",
    "    return y "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weight(file_path):\n",
    "    f = h5py.File(file_path,'r')\n",
    "    \n",
    "    embed = f['embedding']['embedding']['embeddings:0'][:]\n",
    "    \n",
    "    forword_gru_bias = f['bidirectional']['bidirectional']['forward_gru']['gru_cell_1']['bias:0'][:]\n",
    "    forword_gru_kernel = f['bidirectional']['bidirectional']['forward_gru']['gru_cell_1']['kernel:0'][:]\n",
    "    forword_gru_recurrent_kernel = f['bidirectional']['bidirectional']['forward_gru']['gru_cell_1']['recurrent_kernel:0'][:]\n",
    "    \n",
    "    backward_gru_bias = f['bidirectional']['bidirectional']['backward_gru']['gru_cell_2']['bias:0'][:]\n",
    "    backward_gru_kernel = f['bidirectional']['bidirectional']['backward_gru']['gru_cell_2']['kernel:0'][:]\n",
    "    backward_gru_recurrent_kernel = f['bidirectional']['bidirectional']['backward_gru']['gru_cell_2']['recurrent_kernel:0'][:]\n",
    "    \n",
    "    layer_normal_beta = f['layer_normalization']['layer_normalization']['beta:0'][:]\n",
    "    layer_normal_gamma = f['layer_normalization']['layer_normalization']['gamma:0'][:]\n",
    "    \n",
    "    layer_normal_beta1 = f['layer_normalization_1']['layer_normalization_1']['beta:0'][:]\n",
    "    layer_normal_gamma1 = f['layer_normalization_1']['layer_normalization_1']['gamma:0'][:]\n",
    "    \n",
    "    pre_intent_bias = f['pre_intent']['pre_intent']['bias:0'][:]\n",
    "    pre_intent_gamma = f['pre_intent']['pre_intent']['kernel:0'][:]\n",
    "    \n",
    "    pre_ner_bias = f['pre_ner']['pre_ner']['bias:0'][:]\n",
    "    pre_ner_gamma = f['pre_ner']['pre_ner']['kernel:0'][:]\n",
    "    \n",
    "    return embed, forword_gru_bias,forword_gru_kernel,forword_gru_recurrent_kernel, backward_gru_bias,backward_gru_kernel,backward_gru_recurrent_kernel,\\\n",
    "            layer_normal_beta,layer_normal_gamma,layer_normal_beta1,layer_normal_gamma1,pre_intent_bias,pre_intent_gamma,pre_ner_bias,pre_ner_gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trans2labelid(vocab, labels, max_sent_len):\n",
    "    labels = [vocab[label] for label in labels]\n",
    "    if len(labels) < max_sent_len:\n",
    "        labels += [0] * (max_sent_len - len(labels))\n",
    "    else:\n",
    "        labels = labels[:max_sent_len]\n",
    "    return labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(inputs,h5file_path):\n",
    "    embed, forword_gru_bias,forword_gru_kernel,forword_gru_recurrent_kernel, backward_gru_bias,backward_gru_kernel,backward_gru_recurrent_kernel,\\\n",
    "            layer_normal_beta,layer_normal_gammma,layer_normal_beta1,layer_normal_gamma1,pre_intent_bias,pre_intent_gamma,pre_ner_bias,pre_ner_gamma = get_weight(h5file_path)\n",
    "    forword_gru_state = np.zeros((64))\n",
    "    backed_gru_state = np.zeros((64))\n",
    "    x = trans2labelid(char2id,inputs,params['max_sent_len'])\n",
    "    print(x)\n",
    "    embed = embedding(x,params['embed_size'], embed)\n",
    "#     print('embed: {}'.format(embed))\n",
    "    recurrent_embed = embed[::-1]\n",
    "    forword_gru = []\n",
    "    backed_gru = []\n",
    "#     print(np.shape(recurrent_embed))\n",
    "    for i in range(len(x)):\n",
    "#         print(np.shape(embed[i]))\n",
    "        forward_h = GRU(embed[i],forword_gru_state,forword_gru_kernel,forword_gru_recurrent_kernel,forword_gru_bias)\n",
    "        backed_h =  GRU(recurrent_embed[i], backed_gru_state,backward_gru_kernel,backward_gru_recurrent_kernel,backward_gru_bias)\n",
    "        forword_gru_state = forward_h\n",
    "        backed_gru_state = backed_h\n",
    "        forword_gru.append(forward_h)\n",
    "        backed_gru.append(backed_h)\n",
    "    gru_out = np.concatenate((forword_gru,backed_gru[::-1]), axis=-1)\n",
    "#     print('gru_out: {}'.format(gru_out))\n",
    "#     print(np.shape(gru_out))\n",
    "    x_in = LayerNormalization(gru_out,layer_normal_gammma,layer_normal_beta)\n",
    "    print(np.shape(x_in))\n",
    "    print('x_in : ',x_in)\n",
    "    x_conv = GlobalAveragePooling1D(x_in)\n",
    "    print(np.shape(x_conv))\n",
    "    pre_intent = dense(x_conv,pre_intent_gamma,pre_intent_bias)\n",
    "    pre_intent = sigmoid(pre_intent)\n",
    "    \n",
    "    x_ner = LayerNormalization(gru_out,layer_normal_gamma1,layer_normal_beta1)\n",
    "    pre_slot = dense(x_ner,pre_ner_gamma,pre_ner_bias)\n",
    "    pre_slot = sigmoid(pre_slot)\n",
    "    \n",
    "    return embed,gru_out,x_in,x_conv,x_ner,pre_intent,pre_slot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = '打开空调'\n",
    "h5file_path = '../ner_model_weight/model_conv_625.h5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[111, 196, 182, 95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "(20, 128)\n",
      "x_in :  [[-0.18487395  2.59982342  0.08869394 ... -0.6315285   0.83122395\n",
      "   2.1741613 ]\n",
      " [ 0.23971886  2.46877678  0.34089185 ...  0.08862706 -1.99671927\n",
      "   3.54785764]\n",
      " [ 0.05058586  0.85279782 -0.05753813 ... -1.72621784  0.79041573\n",
      "   3.30894318]\n",
      " ...\n",
      " [ 0.67766895  0.88074969  0.44668887 ... -0.35083664  1.60386768\n",
      "   0.19373351]\n",
      " [ 0.68363193  0.8517546   0.41826727 ... -0.12744785  1.64226132\n",
      "   0.26700695]\n",
      " [ 0.70110929  0.83443363  0.39363473 ...  0.23800833  1.40505977\n",
      "   0.46103015]]\n",
      "(128,)\n"
     ]
    }
   ],
   "source": [
    "embed_np,gru_out_np,x_in_np,x_conv_np,x_ner_np,pre_intent_np,pre_slot_np = test(inputs,h5file_path) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 128)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-3.32076249e-02,  4.08698129e-01,  1.88430461e-02, ...,\n",
       "         6.60540134e-04,  1.38070119e-01,  3.86093677e-01],\n",
       "       [ 3.53702873e-03,  2.61830424e-01,  1.61371466e-02, ...,\n",
       "         6.02562128e-02, -2.79264420e-01,  4.60063271e-01],\n",
       "       [-2.86388978e-02,  6.55365384e-02, -5.13157126e-02, ...,\n",
       "        -2.13634347e-01,  7.67432481e-02,  4.97659814e-01],\n",
       "       ...,\n",
       "       [ 4.48726121e-02,  3.06398390e-02,  5.03202754e-03, ...,\n",
       "        -2.66125565e-02,  1.36799462e-01, -5.30799461e-02],\n",
       "       [ 4.74123010e-02,  2.86414116e-02,  2.49992156e-03, ...,\n",
       "         5.39434391e-03,  1.42556573e-01, -4.03520903e-02],\n",
       "       [ 4.96300310e-02,  2.67232457e-02, -7.23004060e-05, ...,\n",
       "         5.37701923e-02,  1.10882100e-01, -1.08583088e-02]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(np.shape(gru_out_np))\n",
    "gru_out_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 128)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-0.18487395,  2.59982342,  0.08869394, ..., -0.6315285 ,\n",
       "         0.83122395,  2.1741613 ],\n",
       "       [ 0.23971886,  2.46877678,  0.34089185, ...,  0.08862706,\n",
       "        -1.99671927,  3.54785764],\n",
       "       [ 0.05058586,  0.85279782, -0.05753813, ..., -1.72621784,\n",
       "         0.79041573,  3.30894318],\n",
       "       ...,\n",
       "       [ 0.67766895,  0.88074969,  0.44668887, ..., -0.35083664,\n",
       "         1.60386768,  0.19373351],\n",
       "       [ 0.68363193,  0.8517546 ,  0.41826727, ..., -0.12744785,\n",
       "         1.64226132,  0.26700695],\n",
       "       [ 0.70110929,  0.83443363,  0.39363473, ...,  0.23800833,\n",
       "         1.40505977,  0.46103015]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(np.shape(x_in_np))\n",
    "x_in_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.97680503e-01,  1.01371085e+00,  4.57867960e-01,  7.11893830e-01,\n",
       "       -2.75346629e+00, -6.72380333e-01,  7.28192348e-01, -7.21241505e-01,\n",
       "       -2.65741050e+00,  2.63547493e+00,  9.37658362e-01, -4.10543900e+00,\n",
       "        1.09610905e-01,  4.65862201e-01,  3.00453949e+00, -1.88779374e+00,\n",
       "       -1.68574877e+00, -1.97146415e+00, -1.23703313e+00,  5.82327613e+00,\n",
       "        4.93449139e-01,  2.27705570e+00, -3.57668323e+00,  1.00738266e-01,\n",
       "       -3.11463523e+00, -1.91158546e+00,  1.08983106e-01, -2.91055873e-01,\n",
       "        6.90614525e-01, -1.92939480e+00, -2.79640531e-01, -5.82465525e-01,\n",
       "        1.93135087e+00,  1.45356317e+00, -2.03315035e+00,  6.04236590e-01,\n",
       "        6.39504497e-01,  4.92324552e+00, -2.56432582e+00,  3.20878610e+00,\n",
       "        2.06785049e-01, -2.05023691e+00, -9.69418706e-01, -3.26773965e-01,\n",
       "        1.58392802e+00, -4.08546109e-01, -2.55674181e+00, -1.69063124e+00,\n",
       "       -5.15720992e-02,  2.37055179e+00,  4.93634195e-01, -2.74292841e+00,\n",
       "       -3.59134881e-01, -3.01090884e+00, -1.35480469e+00,  1.40781610e+00,\n",
       "        1.35099936e+00, -2.70461297e+00,  6.52019246e-01,  2.25583049e+00,\n",
       "        2.79069211e+00, -1.01243850e+00, -8.85120379e-01,  1.51674839e-01,\n",
       "       -1.61406568e-01,  1.38343936e+00, -7.97014292e-02, -3.54160624e-01,\n",
       "       -9.54253636e-01, -2.27316132e+00, -6.78184626e-01, -7.20483705e-01,\n",
       "       -1.29204375e+00,  2.76938865e+00, -6.69819676e-01, -6.75582011e-01,\n",
       "       -6.82106491e-01, -7.31013112e-01, -2.15576782e+00, -1.22038647e+00,\n",
       "        7.23095243e-01,  1.23036664e+00,  7.03055446e-02, -7.09586430e-01,\n",
       "       -3.85624085e-01,  4.24403919e-01,  8.73335404e-01,  3.43015378e-01,\n",
       "       -9.03521518e-01,  7.90360891e-01, -1.32216372e+00,  1.18204132e+00,\n",
       "       -6.79305085e-02,  2.24110694e+00,  9.93691679e-01,  4.19513264e+00,\n",
       "        2.62420600e-01,  9.37789093e-01,  1.58418707e+00, -1.08951882e+00,\n",
       "        6.48036068e-01,  3.77481675e-01, -2.82929030e-01,  1.36709970e+00,\n",
       "        7.73449153e-01,  1.35308892e-01, -2.33933737e-01, -8.96684013e-01,\n",
       "       -1.32803077e+00,  1.48104823e+00,  1.50004496e-01,  1.66818176e+00,\n",
       "        1.47644787e+00, -7.29984775e-01,  1.44774412e+00,  9.53155115e-01,\n",
       "        2.72725637e-01, -4.89267446e-03,  5.08624734e-01,  4.51097883e-01,\n",
       "        1.20728419e+00, -2.46714842e-01,  1.45289824e+00,  6.64853460e-01,\n",
       "        5.10406180e-01, -4.48990736e-01,  1.29374960e+00,  6.46138011e-01])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_conv_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.13309367,  4.43989236,  0.6735068 , ..., -1.25604107,\n",
       "         0.96779568,  3.90535502],\n",
       "       [ 0.69618359,  4.22298717,  1.02304522, ..., -0.09027992,\n",
       "        -2.14163345,  6.3434138 ],\n",
       "       [ 0.44535765,  1.54825818,  0.47083372, ..., -3.02808363,\n",
       "         0.92292552,  5.9193845 ],\n",
       "       ...,\n",
       "       [ 1.27698777,  1.59452344,  1.1696766 , ..., -0.80166748,\n",
       "         1.81734629,  0.39045932],\n",
       "       [ 1.2848958 ,  1.54653148,  1.13028515, ..., -0.44005397,\n",
       "         1.85956153,  0.52050622],\n",
       "       [ 1.30807407,  1.51786223,  1.09614521, ...,  0.15153295,\n",
       "         1.59874957,  0.86486178]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_ner_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.26486299e-11, 3.44658727e-11, 7.77707427e-09, 1.26930422e-11,\n",
       "       2.39740630e-09, 2.37031140e-08, 2.14346056e-08, 4.69827012e-11,\n",
       "       1.30352383e-09, 1.77831338e-05, 3.72474467e-18, 9.79177235e-09,\n",
       "       2.65465432e-08, 3.74631898e-17, 2.47004340e-07, 3.63699069e-16,\n",
       "       3.01376151e-11, 2.97858980e-11, 5.10297266e-07, 1.29272860e-09,\n",
       "       5.80643838e-07, 5.29078474e-12, 4.19446210e-11, 3.20406996e-09,\n",
       "       7.69364845e-08, 5.62570002e-08, 7.43173911e-16, 7.21122280e-15,\n",
       "       5.39827637e-05, 8.54081832e-12, 2.37959413e-11, 1.57130502e-07,\n",
       "       1.58208528e-10, 4.44693206e-13, 7.96728843e-11, 9.69827632e-15,\n",
       "       4.85809455e-10, 2.56163623e-10, 1.71254769e-13, 9.24184487e-12,\n",
       "       1.18567958e-14, 1.68738497e-09, 7.36440718e-12, 9.47962585e-11,\n",
       "       5.03368737e-11, 1.62653677e-16, 1.49624083e-11, 4.66498586e-10,\n",
       "       2.77016688e-12, 5.28493109e-12, 8.74386028e-13, 6.55696446e-11,\n",
       "       7.77618964e-11, 1.03084342e-11, 3.62505197e-11])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_intent_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.41983295e-10, 1.98325884e-10, 3.42983686e-13, 3.25632466e-14,\n",
       "        9.06448294e-15, 1.37813124e-06, 4.37848481e-10, 1.64528917e-11,\n",
       "        4.15654891e-12, 1.11682936e-11, 3.35575006e-11, 1.86729512e-15,\n",
       "        3.08152763e-15, 5.01298455e-15, 1.60529039e-11, 1.78041051e-11,\n",
       "        2.07299795e-16, 8.21172541e-12, 3.94146684e-15, 4.24800101e-10,\n",
       "        7.05444796e-10, 1.53546333e-09, 1.93005401e-14, 3.92427647e-14,\n",
       "        3.19460734e-12, 3.59195040e-11, 1.75251329e-12, 4.98750425e-10,\n",
       "        5.12518727e-11, 4.04607417e-11, 1.62623767e-11, 6.31760309e-12,\n",
       "        9.99843593e-01, 6.92694309e-12, 1.06972354e-07, 2.61576098e-09],\n",
       "       [1.81301288e-08, 7.26018990e-11, 5.97766727e-25, 6.18280565e-17,\n",
       "        1.11609012e-14, 1.94389906e-12, 4.63472514e-15, 5.34867266e-12,\n",
       "        7.53658391e-16, 3.90022717e-21, 5.96289851e-20, 1.54501844e-10,\n",
       "        5.73061121e-18, 9.27800754e-17, 6.87489875e-09, 8.02665226e-17,\n",
       "        1.68309607e-19, 4.05668552e-17, 1.15049796e-19, 1.62079808e-18,\n",
       "        9.30888278e-11, 1.65494857e-24, 4.94095019e-15, 2.06844169e-16,\n",
       "        1.49077606e-18, 3.77237451e-18, 9.35501166e-12, 3.28972171e-20,\n",
       "        4.61945965e-15, 1.11682266e-15, 3.36492595e-13, 2.05285441e-11,\n",
       "        3.08452388e-09, 8.78604627e-12, 8.21350117e-15, 4.02518933e-13],\n",
       "       [5.19374410e-07, 1.55311110e-18, 3.08477594e-12, 1.82036680e-15,\n",
       "        1.95951795e-10, 2.40121295e-12, 4.91543379e-11, 1.00008957e-06,\n",
       "        4.41125787e-14, 1.42313272e-15, 6.10906809e-15, 5.84047257e-10,\n",
       "        3.73777077e-08, 2.10080343e-19, 9.39759811e-17, 3.06729838e-08,\n",
       "        1.54612811e-17, 3.47034612e-18, 9.95681967e-14, 3.86548561e-11,\n",
       "        6.56725213e-14, 3.87300570e-16, 3.21680611e-17, 4.38201044e-13,\n",
       "        1.43017176e-18, 6.28795839e-11, 2.34056275e-14, 3.32065924e-16,\n",
       "        4.46131869e-11, 7.23040202e-09, 2.44797229e-14, 3.16142054e-04,\n",
       "        2.07199949e-08, 3.31622883e-13, 7.10060007e-15, 5.18063719e-13],\n",
       "       [2.04236010e-10, 7.61383129e-12, 4.43491471e-12, 8.64020687e-12,\n",
       "        6.17800573e-07, 1.25972269e-11, 2.83231026e-09, 1.58185686e-13,\n",
       "        9.63549760e-13, 3.59479651e-13, 5.21792320e-13, 1.04313026e-14,\n",
       "        1.22000310e-14, 1.15159535e-08, 2.55011504e-12, 1.14155097e-14,\n",
       "        8.64510387e-17, 1.53451120e-07, 7.71427405e-15, 1.23475591e-11,\n",
       "        3.40886771e-09, 1.89149382e-14, 9.37523637e-11, 1.30154618e-14,\n",
       "        1.35256381e-18, 5.64126882e-14, 6.34746283e-14, 7.73189089e-11,\n",
       "        1.00751795e-04, 1.07190823e-04, 1.58017108e-14, 7.79752138e-15,\n",
       "        6.79263476e-06, 6.79535003e-16, 3.29278806e-16, 9.16167340e-11],\n",
       "       [9.99999853e-01, 2.54393971e-13, 4.59006562e-14, 7.17219008e-14,\n",
       "        1.98940102e-10, 6.68956765e-13, 1.49265926e-12, 9.80751565e-12,\n",
       "        4.31966264e-13, 1.23201601e-14, 2.21395777e-13, 1.09121823e-14,\n",
       "        4.89232057e-20, 2.15319618e-15, 1.30199279e-18, 5.63890276e-13,\n",
       "        1.31387277e-19, 6.15768107e-15, 2.50581416e-17, 1.73148843e-09,\n",
       "        9.49490342e-12, 5.03912630e-13, 6.72879437e-14, 6.34943042e-12,\n",
       "        2.95102805e-16, 1.15913870e-16, 7.85544970e-14, 3.16112662e-11,\n",
       "        1.84274224e-11, 4.98139311e-12, 9.48701843e-16, 1.01355530e-11,\n",
       "        5.46222742e-11, 6.32436484e-16, 6.05636189e-17, 4.95818631e-11],\n",
       "       [9.99999984e-01, 3.26968162e-13, 1.33497599e-15, 9.59993557e-15,\n",
       "        1.47218875e-10, 1.24953766e-13, 7.81770641e-14, 2.11244536e-12,\n",
       "        1.57061328e-14, 1.87608734e-15, 2.42518300e-14, 1.74965048e-17,\n",
       "        2.57524000e-22, 2.47109093e-17, 1.31443716e-19, 3.14255773e-15,\n",
       "        2.48471006e-20, 2.14414567e-17, 7.64026135e-18, 1.06977717e-11,\n",
       "        1.13202786e-11, 3.68172683e-12, 1.77372243e-13, 1.92852274e-12,\n",
       "        3.84743827e-18, 3.46931037e-17, 5.98057096e-14, 1.47945555e-13,\n",
       "        2.87948487e-13, 5.18003345e-14, 3.34725083e-16, 2.11738847e-13,\n",
       "        1.56760202e-12, 8.70582102e-17, 4.33008923e-17, 1.37659161e-12],\n",
       "       [9.99999992e-01, 6.62275315e-13, 2.11916106e-16, 2.40312595e-15,\n",
       "        8.46547381e-11, 7.68291985e-14, 4.61178893e-14, 2.47445798e-12,\n",
       "        3.22154442e-15, 2.79625740e-16, 5.73631382e-15, 2.56924650e-18,\n",
       "        6.91773974e-23, 2.96818607e-18, 1.35740793e-19, 5.87973146e-16,\n",
       "        3.30981045e-20, 1.21194908e-18, 2.52896467e-18, 1.89322961e-12,\n",
       "        1.29307362e-11, 1.98334700e-12, 1.87440582e-13, 9.73442282e-13,\n",
       "        1.04786655e-18, 4.46139738e-17, 5.14564593e-14, 1.72473114e-14,\n",
       "        2.76310666e-14, 1.02001956e-14, 2.12250427e-16, 3.49644246e-14,\n",
       "        1.67218068e-12, 5.01416783e-17, 5.01812769e-17, 2.84557574e-13],\n",
       "       [9.99999995e-01, 1.11952210e-12, 7.00330344e-17, 9.44846109e-16,\n",
       "        5.12474216e-11, 7.25669986e-14, 4.02149815e-14, 4.36072714e-12,\n",
       "        1.60715140e-15, 7.39918802e-17, 2.53644335e-15, 2.22186664e-18,\n",
       "        4.89256221e-23, 9.19526812e-19, 2.10199919e-19, 4.21722111e-16,\n",
       "        6.74996328e-20, 3.10340107e-19, 1.33547051e-18, 1.70467858e-12,\n",
       "        1.15683205e-11, 7.63521803e-13, 2.16716925e-13, 7.73091861e-13,\n",
       "        6.50779025e-19, 6.47814183e-17, 3.69045781e-14, 5.66916615e-15,\n",
       "        8.02542745e-15, 4.76931400e-15, 1.87857618e-16, 1.36332796e-14,\n",
       "        2.80778342e-12, 3.78806344e-17, 5.57752632e-17, 1.38977229e-13],\n",
       "       [9.99999997e-01, 1.61421965e-12, 3.55043066e-17, 5.38105266e-16,\n",
       "        3.56857242e-11, 7.75397465e-14, 3.60560415e-14, 7.52870307e-12,\n",
       "        1.22602829e-15, 3.15228804e-17, 1.63907962e-15, 3.11233548e-18,\n",
       "        4.51103843e-23, 4.57188529e-19, 3.04709797e-19, 4.68651376e-16,\n",
       "        1.30104261e-19, 1.69720994e-19, 9.85687578e-19, 2.57874836e-12,\n",
       "        9.52207545e-12, 3.74781110e-13, 2.81919101e-13, 7.82389444e-13,\n",
       "        5.21619070e-19, 8.62708907e-17, 2.62789868e-14, 3.00330402e-15,\n",
       "        4.10291719e-15, 3.13070215e-15, 1.84426716e-16, 8.11960443e-15,\n",
       "        4.11772150e-12, 2.98728423e-17, 6.12527424e-17, 9.85416165e-14],\n",
       "       [9.99999997e-01, 2.12316851e-12, 2.24849158e-17, 3.81178797e-16,\n",
       "        2.70906244e-11, 8.29351583e-14, 3.17776496e-14, 1.16880965e-11,\n",
       "        1.12307262e-15, 1.77766381e-17, 1.29044949e-15, 4.61388498e-18,\n",
       "        4.45727281e-23, 2.90255719e-19, 3.95832798e-19, 5.83559598e-16,\n",
       "        2.19839182e-19, 1.29416755e-19, 8.66612280e-19, 4.24730975e-12,\n",
       "        7.73246270e-12, 2.29070147e-13, 3.72978811e-13, 8.63187689e-13,\n",
       "        4.57696464e-19, 1.06942016e-16, 1.95880963e-14, 2.03875137e-15,\n",
       "        2.67821902e-15, 2.38709190e-15, 1.84693988e-16, 5.91992374e-15,\n",
       "        5.36241344e-12, 2.40308691e-17, 6.67959282e-17, 8.11549805e-14],\n",
       "       [9.99999998e-01, 2.66287222e-12, 1.59619600e-17, 3.02761696e-16,\n",
       "        2.15594543e-11, 8.60986952e-14, 2.79614801e-14, 1.65680559e-11,\n",
       "        1.11391244e-15, 1.17755507e-17, 1.12871043e-15, 6.60544334e-18,\n",
       "        4.52515837e-23, 2.10999221e-19, 4.80802377e-19, 7.35336921e-16,\n",
       "        3.33258814e-19, 1.13428797e-19, 8.32970379e-19, 6.70171952e-12,\n",
       "        6.32461165e-12, 1.59381983e-13, 4.77192249e-13, 9.73684465e-13,\n",
       "        4.20040697e-19, 1.27785992e-16, 1.54505924e-14, 1.57836690e-15,\n",
       "        1.97145581e-15, 1.96738563e-15, 1.85668771e-16, 4.81018854e-15,\n",
       "        6.58784847e-12, 1.99293157e-17, 7.29075861e-17, 7.12679742e-14],\n",
       "       [9.99999998e-01, 3.25347837e-12, 1.21611080e-17, 2.57864636e-16,\n",
       "        1.78003940e-11, 8.73005358e-14, 2.49793470e-14, 2.20404964e-11,\n",
       "        1.15423387e-15, 8.70779904e-18, 1.05714661e-15, 9.14018508e-18,\n",
       "        4.72135081e-23, 1.68324941e-19, 5.67194131e-19, 9.16651905e-16,\n",
       "        4.69501231e-19, 1.06595291e-19, 8.45082411e-19, 9.94704650e-12,\n",
       "        5.26304576e-12, 1.20455844e-13, 5.88362407e-13, 1.10607296e-12,\n",
       "        3.98810824e-19, 1.50760707e-16, 1.28586095e-14, 1.32091402e-15,\n",
       "        1.57467076e-15, 1.71295305e-15, 1.87765914e-16, 4.19783559e-15,\n",
       "        7.86158870e-12, 1.71954779e-17, 8.05289683e-17, 6.51656577e-14],\n",
       "       [9.99999998e-01, 3.91801659e-12, 9.81945581e-18, 2.32635384e-16,\n",
       "        1.52831598e-11, 8.76384783e-14, 2.29441815e-14, 2.80066670e-11,\n",
       "        1.24074111e-15, 7.03663990e-18, 1.04791076e-15, 1.23758126e-17,\n",
       "        5.11193054e-23, 1.45288916e-19, 6.65627834e-19, 1.12972424e-15,\n",
       "        6.33036536e-19, 1.05556440e-19, 8.92195559e-19, 1.40219263e-11,\n",
       "        4.50184147e-12, 9.75599973e-14, 7.11450732e-13, 1.26958681e-12,\n",
       "        3.88474721e-19, 1.78764638e-16, 1.12352263e-14, 1.17071321e-15,\n",
       "        1.35306751e-15, 1.57218109e-15, 1.91579658e-16, 3.86328971e-15,\n",
       "        9.18325688e-12, 1.55559816e-17, 9.09963553e-17, 6.20788207e-14],\n",
       "       [9.99999998e-01, 4.69995977e-12, 8.42684293e-18, 2.20866807e-16,\n",
       "        1.36921497e-11, 8.73510547e-14, 2.19150236e-14, 3.41850005e-11,\n",
       "        1.38912426e-15, 6.10665051e-18, 1.09540712e-15, 1.63196306e-17,\n",
       "        5.78735432e-23, 1.34811427e-19, 7.90928406e-19, 1.37407938e-15,\n",
       "        8.34966390e-19, 1.10117071e-19, 9.75061670e-19, 1.89425060e-11,\n",
       "        3.98842058e-12, 8.48929813e-14, 8.66132802e-13, 1.47789405e-12,\n",
       "        3.78790129e-19, 2.16492036e-16, 1.02445079e-14, 1.08954481e-15,\n",
       "        1.25148695e-15, 1.52731923e-15, 1.97218748e-16, 3.67239966e-15,\n",
       "        1.04497407e-11, 1.48411393e-17, 1.05081940e-16, 6.21375504e-14],\n",
       "       [9.99999998e-01, 5.67576542e-12, 7.76869355e-18, 2.13994099e-16,\n",
       "        1.27077470e-11, 8.41261841e-14, 2.19046876e-14, 3.97001031e-11,\n",
       "        1.60408736e-15, 5.51820052e-18, 1.19404239e-15, 1.98684596e-17,\n",
       "        6.74860971e-23, 1.32047528e-19, 9.59677388e-19, 1.60477482e-15,\n",
       "        1.07866524e-18, 1.20191494e-19, 1.10425374e-18, 2.42658336e-11,\n",
       "        3.61443251e-12, 7.99292063e-14, 1.08822163e-12, 1.71205912e-12,\n",
       "        3.49717911e-19, 2.69173072e-16, 9.55253770e-15, 1.04314352e-15,\n",
       "        1.22677080e-15, 1.55831901e-15, 2.03636087e-16, 3.46005944e-15,\n",
       "        1.13861856e-11, 1.46880724e-17, 1.19273999e-16, 6.50099455e-14],\n",
       "       [9.99999998e-01, 6.88879402e-12, 7.66446323e-18, 1.93864035e-16,\n",
       "        1.16577691e-11, 7.31477119e-14, 2.22995234e-14, 4.32086896e-11,\n",
       "        1.75563431e-15, 4.88904020e-18, 1.29968813e-15, 1.99032751e-17,\n",
       "        7.52611062e-23, 1.27761338e-19, 1.15912303e-18, 1.64940692e-15,\n",
       "        1.28905859e-18, 1.32193846e-19, 1.28889823e-18, 2.79699124e-11,\n",
       "        3.15992905e-12, 7.94401951e-14, 1.39212852e-12, 1.82728452e-12,\n",
       "        2.84990169e-19, 3.26849495e-16, 8.47694247e-15, 9.52463142e-16,\n",
       "        1.16148465e-15, 1.57990942e-15, 2.07860303e-16, 3.02767010e-15,\n",
       "        1.15333000e-11, 1.37608890e-17, 1.18253988e-16, 6.69497892e-14],\n",
       "       [9.99999997e-01, 8.10895597e-12, 7.54543961e-18, 1.51043923e-16,\n",
       "        9.32047739e-12, 5.47457014e-14, 2.06099957e-14, 4.57197506e-11,\n",
       "        1.48962792e-15, 4.03008118e-18, 1.30840444e-15, 1.64353523e-17,\n",
       "        7.20957192e-23, 1.05445591e-19, 1.28526753e-18, 1.36501200e-15,\n",
       "        1.25544186e-18, 1.40859480e-19, 1.50780673e-18, 2.67935431e-11,\n",
       "        2.46404459e-12, 7.33133370e-14, 1.57132061e-12, 1.59531316e-12,\n",
       "        2.18681938e-19, 3.22522126e-16, 6.15501361e-15, 7.15745192e-16,\n",
       "        8.66766581e-16, 1.39256773e-15, 2.10743292e-16, 2.53338470e-15,\n",
       "        1.06495527e-11, 1.01455339e-17, 8.64668310e-17, 6.09298759e-14],\n",
       "       [9.99999997e-01, 9.05138320e-12, 6.03297079e-18, 1.33504165e-16,\n",
       "        5.80338412e-12, 4.24133031e-14, 1.41808673e-14, 5.44800539e-11,\n",
       "        9.52514266e-16, 3.65027652e-18, 1.23182944e-15, 2.29683951e-17,\n",
       "        7.01581502e-23, 7.06924003e-20, 1.23637757e-18, 1.20040280e-15,\n",
       "        9.60433766e-19, 1.77062889e-19, 1.84530036e-18, 2.19666326e-11,\n",
       "        1.91763086e-12, 4.96557807e-14, 1.11768835e-12, 1.28392465e-12,\n",
       "        2.71765337e-19, 2.06404331e-16, 3.53877133e-15, 4.39927550e-16,\n",
       "        5.22615793e-16, 9.76025923e-16, 2.30512918e-16, 3.30201649e-15,\n",
       "        9.48773478e-12, 5.98387768e-18, 5.37202380e-17, 5.26279787e-14],\n",
       "       [9.99999998e-01, 9.67038745e-12, 2.95648219e-18, 3.58430588e-16,\n",
       "        4.36757107e-12, 5.15385080e-14, 7.53625204e-15, 8.61862452e-11,\n",
       "        1.14380124e-15, 6.94381971e-18, 1.55664400e-15, 2.38501553e-16,\n",
       "        1.57843456e-22, 7.17850739e-20, 1.22873899e-18, 2.65194593e-15,\n",
       "        7.35607560e-19, 4.79582319e-19, 3.25765592e-18, 2.19380094e-11,\n",
       "        2.53942100e-12, 2.82447595e-14, 4.60600698e-13, 2.09918012e-12,\n",
       "        1.16725002e-18, 1.16489674e-16, 3.11092902e-15, 3.92491435e-16,\n",
       "        8.20781172e-16, 8.59249142e-16, 3.27284369e-16, 1.97359203e-14,\n",
       "        8.28754982e-12, 6.58528967e-18, 6.04907411e-17, 6.84160568e-14],\n",
       "       [9.99999995e-01, 2.40654226e-11, 4.96834124e-19, 5.01773141e-15,\n",
       "        9.66128207e-12, 8.38607280e-14, 3.66530174e-15, 1.24015067e-10,\n",
       "        1.01370812e-14, 9.60995865e-17, 4.44952766e-15, 7.23593342e-15,\n",
       "        4.88527187e-22, 7.18126290e-19, 2.12733025e-18, 1.58194101e-14,\n",
       "        6.04230798e-19, 3.44051096e-18, 2.52878767e-17, 2.17226912e-11,\n",
       "        1.18598579e-11, 2.83262419e-14, 3.22638577e-13, 6.67735062e-12,\n",
       "        5.52333133e-18, 1.77228075e-16, 1.21398855e-14, 1.11506686e-15,\n",
       "        2.59229765e-14, 3.21732667e-15, 8.38827037e-16, 3.33520552e-13,\n",
       "        1.20309930e-11, 2.68037879e-17, 1.81819975e-16, 7.63344527e-14]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_slot_np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 前向与原始模型对比验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['intent_num'] = len(intent2id)\n",
    "params['slot_num'] = len(slot2id)\n",
    "params['id2intent'] = id2intent\n",
    "params['id2slot'] = id2slot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input (InputLayer)              [(None, 20)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 20, 32)       16000       Input[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   (None, 20, 128)      37632       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization (LayerNorma (None, 20, 128)      256         bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d (Globa (None, 128)          0           layer_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "layer_normalization_1 (LayerNor (None, 20, 128)      256         bidirectional[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "pre_intent (Dense)              (None, 55)           7095        global_average_pooling1d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "pre_ner (Dense)                 (None, 20, 36)       4644        layer_normalization_1[0][0]      \n",
      "==================================================================================================\n",
      "Total params: 65,883\n",
      "Trainable params: 65,883\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.keras.backend.clear_session()\n",
    "text_inputs = tf.keras.layers.Input(shape=(20,),name='Input')\n",
    "embed = tf.keras.layers.Embedding(500,32)(text_inputs)\n",
    "bilstm = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(64,return_sequences=True))(embed)\n",
    "x_in = tf.keras.layers.LayerNormalization()(bilstm)\n",
    "x_conv = tf.keras.layers.GlobalAveragePooling1D()(x_in)\n",
    "pre_intent = tf.keras.layers.Dense(params['intent_num'],\\\n",
    "            activation='sigmoid',name = 'pre_intent')(x_conv)\n",
    "x_ner  = tf.keras.layers.LayerNormalization()(bilstm)\n",
    "pre_slot = tf.keras.layers.Dense(params['slot_num'],activation='sigmoid',name = 'pre_ner')(x_ner)\n",
    "model = tf.keras.Model(text_inputs,[embed,bilstm,x_in,x_conv,x_ner,pre_intent,pre_slot])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('../ner_model_weight/model_conv_625.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = trans2labelid(char2id,inputs,params['max_sent_len'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed,bilstm,x_in,x_conv,x_ner,pre_intent,pre_slot = model.predict([x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-0.01878147,  0.17953606,  0.2351896 ,  0.03232408,\n",
       "          0.12217069, -0.2692377 ,  0.23278764, -0.543951  ,\n",
       "         -0.27791595, -0.08053666, -0.21924788,  0.01820837,\n",
       "         -0.1825229 ,  0.6028638 ,  0.19234414, -0.35890377,\n",
       "         -0.28960776, -0.34156418, -0.23981574, -0.02046277,\n",
       "          0.08026391, -0.36386955, -0.4316103 ,  0.15253805,\n",
       "         -0.14915386,  0.08520172, -0.3779107 ,  0.26808628,\n",
       "          0.01118689, -0.21329257,  0.05771829,  0.10948844],\n",
       "        [ 0.08922475,  0.10103845,  0.15886837,  0.24098003,\n",
       "         -0.05900533,  0.16090406,  0.06093986,  0.19829786,\n",
       "          0.14084788, -0.00092293,  0.0456003 ,  0.12744607,\n",
       "          0.02093187, -0.02989241, -0.08713216,  0.03822941,\n",
       "          0.05806618,  0.12983367,  0.07562704,  0.01728018,\n",
       "          0.10335472,  0.33283266,  0.11834561, -0.13072117,\n",
       "         -0.08304434,  0.19359136, -0.24208249, -0.16230007,\n",
       "          0.03459801,  0.00902355,  0.20214981,  0.03549137],\n",
       "        [-0.47242895,  0.11618707, -0.05314695, -0.1335091 ,\n",
       "          0.12577045, -0.10190495, -0.3876142 , -0.35954893,\n",
       "         -0.39866406, -0.15852198,  0.08448235,  0.5798926 ,\n",
       "         -0.12975776,  0.14948176,  0.2137176 ,  0.17682624,\n",
       "          0.26793548,  0.2829737 ,  0.149491  , -0.50894177,\n",
       "          0.05551057,  0.05888956,  0.10430326,  0.15028936,\n",
       "         -0.17902383,  0.01980728,  0.06667099, -0.03593673,\n",
       "          0.01330546,  0.14249316,  0.37644348,  0.32075462],\n",
       "        [-0.04267891, -0.27426872,  0.08768933,  0.01603121,\n",
       "          0.03180897, -0.09006025,  0.02586818,  0.05875162,\n",
       "          0.2604704 , -0.06740848, -0.53260976, -0.2946789 ,\n",
       "          0.34450367, -0.28940362,  0.17108344, -0.13494565,\n",
       "          0.3729624 ,  0.03993977, -0.12544987,  0.1672705 ,\n",
       "         -0.3308428 ,  0.25342968,  0.36547795,  0.13886493,\n",
       "         -0.12818725,  0.27653167,  0.217649  , -0.25566626,\n",
       "         -0.007477  ,  0.06078345, -0.13468069, -0.21515636],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044],\n",
       "        [ 0.11748598, -0.10169111, -0.01604519,  0.05824397,\n",
       "         -0.18862158,  0.0220848 , -0.22570218, -0.04718409,\n",
       "         -0.07866757, -0.24957505,  0.07784698,  0.07966344,\n",
       "         -0.01228018,  0.04383402, -0.00335285,  0.07018775,\n",
       "          0.12504631, -0.08531963,  0.05648921,  0.06789243,\n",
       "          0.20771828,  0.12227602, -0.04382699, -0.09559155,\n",
       "          0.02183615, -0.13216528, -0.10716768, -0.06680635,\n",
       "          0.05916253, -0.03150459, -0.04548346, -0.06439044]]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_in,x_conv,x_nerembed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 20, 128)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[-3.3207625e-02,  4.0869811e-01,  1.8843034e-02, ...,\n",
       "          6.6053419e-04,  1.3807012e-01,  3.8609371e-01],\n",
       "        [ 3.5370304e-03,  2.6183042e-01,  1.6137134e-02, ...,\n",
       "          6.0256232e-02, -2.7926442e-01,  4.6006328e-01],\n",
       "        [-2.8638907e-02,  6.5536484e-02, -5.1315725e-02, ...,\n",
       "         -2.1363437e-01,  7.6743267e-02,  4.9765980e-01],\n",
       "        ...,\n",
       "        [ 4.4872627e-02,  3.0639863e-02,  5.0320509e-03, ...,\n",
       "         -2.6612561e-02,  1.3679947e-01, -5.3079933e-02],\n",
       "        [ 4.7412317e-02,  2.8641418e-02,  2.4999392e-03, ...,\n",
       "          5.3943517e-03,  1.4255656e-01, -4.0352091e-02],\n",
       "        [ 4.9630046e-02,  2.6723254e-02, -7.2289338e-05, ...,\n",
       "          5.3770192e-02,  1.1088210e-01, -1.0858308e-02]]], dtype=float32)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(bilstm.shape)\n",
    "bilstm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 20, 128)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[[-0.18487397,  2.5998235 ,  0.08869389, ..., -0.6315285 ,\n",
       "          0.831224  ,  2.1741617 ],\n",
       "        [ 0.23971885,  2.4687767 ,  0.34089172, ...,  0.08862716,\n",
       "         -1.9967191 ,  3.5478573 ],\n",
       "        [ 0.05058582,  0.85279745, -0.0575382 , ..., -1.726218  ,\n",
       "          0.7904159 ,  3.308943  ],\n",
       "        ...,\n",
       "        [ 0.67766905,  0.8807498 ,  0.446689  , ..., -0.3508367 ,\n",
       "          1.6038677 ,  0.19373357],\n",
       "        [ 0.683632  ,  0.85175467,  0.41826737, ..., -0.12744778,\n",
       "          1.6422613 ,  0.26700693],\n",
       "        [ 0.7011094 ,  0.8344337 ,  0.3936348 , ...,  0.23800832,\n",
       "          1.4050599 ,  0.4610302 ]]], dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x_in.shape)\n",
    "x_in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "array([[-0.18487395,  2.59982342,  0.08869394, ..., -0.6315285 ,\n",
    "         0.83122395,  2.1741613 ],\n",
    "       [ 0.23971886,  2.46877678,  0.34089185, ...,  0.08862706,\n",
    "        -1.99671927,  3.54785764],\n",
    "       [ 0.05058586,  0.85279782, -0.05753813, ..., -1.72621784,\n",
    "         0.79041573,  3.30894318],\n",
    "       ...,\n",
    "       [ 0.67766895,  0.88074969,  0.44668887, ..., -0.35083664,\n",
    "         1.60386768,  0.19373351],\n",
    "       [ 0.68363193,  0.8517546 ,  0.41826727, ..., -0.12744785,\n",
    "         1.64226132,  0.26700695],\n",
    "       [ 0.70110929,  0.83443363,  0.39363473, ...,  0.23800833,\n",
    "         1.40505977,  0.46103015]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 128)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_conv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.97680521e-01,  1.01371109e+00,  4.57867950e-01,\n",
       "         7.11894035e-01, -2.75346613e+00, -6.72380149e-01,\n",
       "         7.28192210e-01, -7.21241653e-01, -2.65741014e+00,\n",
       "         2.63547468e+00,  9.37658310e-01, -4.10543919e+00,\n",
       "         1.09610513e-01,  4.65861976e-01,  3.00453949e+00,\n",
       "        -1.88779378e+00, -1.68574882e+00, -1.97146380e+00,\n",
       "        -1.23703337e+00,  5.82327604e+00,  4.93448883e-01,\n",
       "         2.27705550e+00, -3.57668352e+00,  1.00738250e-01,\n",
       "        -3.11463499e+00, -1.91158509e+00,  1.08983144e-01,\n",
       "        -2.91056186e-01,  6.90614522e-01, -1.92939472e+00,\n",
       "        -2.79640317e-01, -5.82465708e-01,  1.93135095e+00,\n",
       "         1.45356345e+00, -2.03315067e+00,  6.04236484e-01,\n",
       "         6.39504790e-01,  4.92324543e+00, -2.56432581e+00,\n",
       "         3.20878553e+00,  2.06785053e-01, -2.05023670e+00,\n",
       "        -9.69418705e-01, -3.26773494e-01,  1.58392823e+00,\n",
       "        -4.08545971e-01, -2.55674171e+00, -1.69063127e+00,\n",
       "        -5.15721440e-02,  2.37055159e+00,  4.93634850e-01,\n",
       "        -2.74292850e+00, -3.59134853e-01, -3.01090932e+00,\n",
       "        -1.35480475e+00,  1.40781593e+00,  1.35099936e+00,\n",
       "        -2.70461345e+00,  6.52019322e-01,  2.25583029e+00,\n",
       "         2.79069185e+00, -1.01243854e+00, -8.85120392e-01,\n",
       "         1.51674807e-01, -1.61406532e-01,  1.38343942e+00,\n",
       "        -7.97014013e-02, -3.54160607e-01, -9.54253793e-01,\n",
       "        -2.27316141e+00, -6.78184748e-01, -7.20483601e-01,\n",
       "        -1.29204357e+00,  2.76938844e+00, -6.69819713e-01,\n",
       "        -6.75582051e-01, -6.82106614e-01, -7.31013179e-01,\n",
       "        -2.15576816e+00, -1.22038651e+00,  7.23095179e-01,\n",
       "         1.23036683e+00,  7.03055412e-02, -7.09586501e-01,\n",
       "        -3.85624081e-01,  4.24403846e-01,  8.73335361e-01,\n",
       "         3.43015373e-01, -9.03521657e-01,  7.90360868e-01,\n",
       "        -1.32216370e+00,  1.18204141e+00, -6.79305494e-02,\n",
       "         2.24110723e+00,  9.93691742e-01,  4.19513273e+00,\n",
       "         2.62420595e-01,  9.37788963e-01,  1.58418727e+00,\n",
       "        -1.08951867e+00,  6.48036122e-01,  3.77481639e-01,\n",
       "        -2.82929003e-01,  1.36710000e+00,  7.73449063e-01,\n",
       "         1.35308892e-01, -2.33933762e-01, -8.96684051e-01,\n",
       "        -1.32803094e+00,  1.48104835e+00,  1.50004447e-01,\n",
       "         1.66818178e+00,  1.47644782e+00, -7.29984879e-01,\n",
       "         1.44774401e+00,  9.53155041e-01,  2.72725612e-01,\n",
       "        -4.89264121e-03,  5.08624673e-01,  4.51097906e-01,\n",
       "         1.20728433e+00, -2.46714920e-01,  1.45289826e+00,\n",
       "         6.64853454e-01,  5.10406137e-01, -4.48990732e-01,\n",
       "         1.29374957e+00,  6.46137953e-01]], dtype=float32)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.13309366,  4.4398923 ,  0.67350674, ..., -1.256041  ,\n",
       "          0.9677957 ,  3.9053555 ],\n",
       "        [ 0.69618356,  4.222987  ,  1.0230451 , ..., -0.09027976,\n",
       "         -2.1416333 ,  6.3434134 ],\n",
       "        [ 0.44535762,  1.5482576 ,  0.47083363, ..., -3.0280838 ,\n",
       "          0.9229257 ,  5.919384  ],\n",
       "        ...,\n",
       "        [ 1.2769878 ,  1.5945237 ,  1.1696768 , ..., -0.8016676 ,\n",
       "          1.8173462 ,  0.39045942],\n",
       "        [ 1.2848959 ,  1.5465316 ,  1.1302853 , ..., -0.44005388,\n",
       "          1.8595614 ,  0.5205062 ],\n",
       "        [ 1.3080742 ,  1.5178623 ,  1.0961454 , ...,  0.15153301,\n",
       "          1.5987498 ,  0.86486185]]], dtype=float32)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_ner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 20, 128)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(bilstm_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.2648812e-11, 3.4465885e-11, 7.7770652e-09, 1.2693058e-11,\n",
       "        2.3974007e-09, 2.3703075e-08, 2.1434621e-08, 4.6982664e-11,\n",
       "        1.3035236e-09, 1.7783097e-05, 3.7247546e-18, 9.7917763e-09,\n",
       "        2.6546489e-08, 3.7463165e-17, 2.4700412e-07, 3.6369755e-16,\n",
       "        3.0137576e-11, 2.9785951e-11, 5.1029684e-07, 1.2927308e-09,\n",
       "        5.8064313e-07, 5.2907899e-12, 4.1944562e-11, 3.2040588e-09,\n",
       "        7.6936509e-08, 5.6257043e-08, 7.4317374e-16, 7.2112014e-15,\n",
       "        5.3982822e-05, 8.5408043e-12, 2.3795993e-11, 1.5713033e-07,\n",
       "        1.5820807e-10, 4.4469354e-13, 7.9673032e-11, 9.6982918e-15,\n",
       "        4.8580800e-10, 2.5616401e-10, 1.7125476e-13, 9.2418590e-12,\n",
       "        1.1856786e-14, 1.6873805e-09, 7.3644025e-12, 9.4796331e-11,\n",
       "        5.0336926e-11, 1.6265312e-16, 1.4962413e-11, 4.6649712e-10,\n",
       "        2.7701658e-12, 5.2849400e-12, 8.7439057e-13, 6.5569758e-11,\n",
       "        7.7761679e-11, 1.0308426e-11, 3.6250520e-11]], dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_intent_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[3.41982748e-10, 1.98325759e-10, 3.42984294e-13, 3.25633245e-14,\n",
       "         9.06449174e-15, 1.37813151e-06, 4.37848285e-10, 1.64528790e-11,\n",
       "         4.15655227e-12, 1.11683050e-11, 3.35575352e-11, 1.86729112e-15,\n",
       "         3.08153297e-15, 5.01299636e-15, 1.60529090e-11, 1.78041974e-11,\n",
       "         2.07299786e-16, 8.21171516e-12, 3.94146114e-15, 4.24800695e-10,\n",
       "         7.05447090e-10, 1.53546631e-09, 1.93005300e-14, 3.92427756e-14,\n",
       "         3.19460756e-12, 3.59195347e-11, 1.75251849e-12, 4.98752539e-10,\n",
       "         5.12518604e-11, 4.04608187e-11, 1.62623751e-11, 6.31762481e-12,\n",
       "         9.99843597e-01, 6.92695684e-12, 1.06972749e-07, 2.61576516e-09],\n",
       "        [1.81301196e-08, 7.26020910e-11, 5.97769508e-25, 6.18278465e-17,\n",
       "         1.11609203e-14, 1.94389838e-12, 4.63470764e-15, 5.34866552e-12,\n",
       "         7.53659953e-16, 3.90024343e-21, 5.96290658e-20, 1.54501925e-10,\n",
       "         5.73060637e-18, 9.27806009e-17, 6.87492285e-09, 8.02664369e-17,\n",
       "         1.68309683e-19, 4.05668728e-17, 1.15050328e-19, 1.62080654e-18,\n",
       "         9.30889602e-11, 1.65495820e-24, 4.94094689e-15, 2.06844784e-16,\n",
       "         1.49078367e-18, 3.77237125e-18, 9.35502717e-12, 3.28971507e-20,\n",
       "         4.61943944e-15, 1.11681961e-15, 3.36492824e-13, 2.05285355e-11,\n",
       "         3.08452974e-09, 8.78603874e-12, 8.21352009e-15, 4.02519516e-13],\n",
       "        [5.19374908e-07, 1.55311163e-18, 3.08478416e-12, 1.82036528e-15,\n",
       "         1.95951630e-10, 2.40121356e-12, 4.91543681e-11, 1.00008947e-06,\n",
       "         4.41124120e-14, 1.42313658e-15, 6.10905742e-15, 5.84046977e-10,\n",
       "         3.73776849e-08, 2.10079370e-19, 9.39760225e-17, 3.06729468e-08,\n",
       "         1.54613093e-17, 3.47032997e-18, 9.95680724e-14, 3.86548502e-11,\n",
       "         6.56725447e-14, 3.87300230e-16, 3.21680085e-17, 4.38201070e-13,\n",
       "         1.43017677e-18, 6.28795904e-11, 2.34055939e-14, 3.32064973e-16,\n",
       "         4.46130598e-11, 7.23039451e-09, 2.44797316e-14, 3.16141726e-04,\n",
       "         2.07200195e-08, 3.31622425e-13, 7.10059419e-15, 5.18064216e-13],\n",
       "        [2.04236433e-10, 7.61384185e-12, 4.43492682e-12, 8.64019446e-12,\n",
       "         6.17800822e-07, 1.25972106e-11, 2.83232193e-09, 1.58185788e-13,\n",
       "         9.63548360e-13, 3.59479400e-13, 5.21792895e-13, 1.04312954e-14,\n",
       "         1.22000730e-14, 1.15159429e-08, 2.55012287e-12, 1.14155299e-14,\n",
       "         8.64510149e-17, 1.53450742e-07, 7.71426956e-15, 1.23475292e-11,\n",
       "         3.40886919e-09, 1.89149911e-14, 9.37523462e-11, 1.30154769e-14,\n",
       "         1.35256686e-18, 5.64125671e-14, 6.34746433e-14, 7.73187347e-11,\n",
       "         1.00751953e-04, 1.07191008e-04, 1.58016859e-14, 7.79755407e-15,\n",
       "         6.79265395e-06, 6.79536099e-16, 3.29278526e-16, 9.16165685e-11],\n",
       "        [9.99999881e-01, 2.54393755e-13, 4.59007222e-14, 7.17217965e-14,\n",
       "         1.98940656e-10, 6.68957240e-13, 1.49266331e-12, 9.80751851e-12,\n",
       "         4.31965634e-13, 1.23201738e-14, 2.21394937e-13, 1.09121967e-14,\n",
       "         4.89232096e-20, 2.15319439e-15, 1.30199697e-18, 5.63891436e-13,\n",
       "         1.31387094e-19, 6.15765212e-15, 2.50582058e-17, 1.73148917e-09,\n",
       "         9.49490139e-12, 5.03913426e-13, 6.72879179e-14, 6.34943226e-12,\n",
       "         2.95102547e-16, 1.15913931e-16, 7.85544793e-14, 3.16112102e-11,\n",
       "         1.84274384e-11, 4.98139811e-12, 9.48700724e-16, 1.01355703e-11,\n",
       "         5.46222928e-11, 6.32436356e-16, 6.05637954e-17, 4.95817692e-11],\n",
       "        [1.00000000e+00, 3.26969680e-13, 1.33498163e-15, 9.59990466e-15,\n",
       "         1.47219098e-10, 1.24953772e-13, 7.81770685e-14, 2.11244520e-12,\n",
       "         1.57061033e-14, 1.87609476e-15, 2.42518221e-14, 1.74965160e-17,\n",
       "         2.57524842e-22, 2.47109686e-17, 1.31444247e-19, 3.14256254e-15,\n",
       "         2.48472325e-20, 2.14413487e-17, 7.64025125e-18, 1.06977855e-11,\n",
       "         1.13202746e-11, 3.68172203e-12, 1.77372006e-13, 1.92852440e-12,\n",
       "         3.84744988e-18, 3.46931262e-17, 5.98056828e-14, 1.47945417e-13,\n",
       "         2.87948241e-13, 5.18002660e-14, 3.34725583e-16, 2.11738965e-13,\n",
       "         1.56760401e-12, 8.70582845e-17, 4.33009860e-17, 1.37659101e-12],\n",
       "        [1.00000000e+00, 6.62274326e-13, 2.11915626e-16, 2.40311590e-15,\n",
       "         8.46547416e-11, 7.68290799e-14, 4.61179218e-14, 2.47445619e-12,\n",
       "         3.22154434e-15, 2.79625666e-16, 5.73630194e-15, 2.56924821e-18,\n",
       "         6.91774153e-23, 2.96818336e-18, 1.35740946e-19, 5.87975120e-16,\n",
       "         3.30981717e-20, 1.21194760e-18, 2.52896245e-18, 1.89322993e-12,\n",
       "         1.29307242e-11, 1.98334621e-12, 1.87440490e-13, 9.73444090e-13,\n",
       "         1.04786547e-18, 4.46140327e-17, 5.14564011e-14, 1.72472459e-14,\n",
       "         2.76311295e-14, 1.02001952e-14, 2.12249741e-16, 3.49644799e-14,\n",
       "         1.67218182e-12, 5.01415778e-17, 5.01811931e-17, 2.84558239e-13],\n",
       "        [1.00000000e+00, 1.11952353e-12, 7.00331341e-17, 9.44843442e-16,\n",
       "         5.12474611e-11, 7.25668237e-14, 4.02149458e-14, 4.36071838e-12,\n",
       "         1.60714708e-15, 7.39920271e-17, 2.53643465e-15, 2.22186275e-18,\n",
       "         4.89253742e-23, 9.19524885e-19, 2.10199609e-19, 4.21721161e-16,\n",
       "         6.74997991e-20, 3.10338650e-19, 1.33546345e-18, 1.70467817e-12,\n",
       "         1.15683210e-11, 7.63521570e-13, 2.16716402e-13, 7.73090576e-13,\n",
       "         6.50778402e-19, 6.47814305e-17, 3.69045005e-14, 5.66914747e-15,\n",
       "         8.02539662e-15, 4.76931388e-15, 1.87857892e-16, 1.36332629e-14,\n",
       "         2.80779068e-12, 3.78806270e-17, 5.57752468e-17, 1.38977073e-13],\n",
       "        [1.00000000e+00, 1.61422156e-12, 3.55043356e-17, 5.38103038e-16,\n",
       "         3.56857183e-11, 7.75398422e-14, 3.60560750e-14, 7.52869815e-12,\n",
       "         1.22602979e-15, 3.15228473e-17, 1.63907641e-15, 3.11233861e-18,\n",
       "         4.51106985e-23, 4.57188670e-19, 3.04710177e-19, 4.68651259e-16,\n",
       "         1.30104809e-19, 1.69721021e-19, 9.85686099e-19, 2.57875145e-12,\n",
       "         9.52208624e-12, 3.74780339e-13, 2.81918152e-13, 7.82388640e-13,\n",
       "         5.21620146e-19, 8.62708152e-17, 2.62790498e-14, 3.00330206e-15,\n",
       "         4.10291367e-15, 3.13070514e-15, 1.84427594e-16, 8.11959515e-15,\n",
       "         4.11772135e-12, 2.98729098e-17, 6.12529295e-17, 9.85418005e-14],\n",
       "        [1.00000000e+00, 2.12316991e-12, 2.24848320e-17, 3.81179411e-16,\n",
       "         2.70906388e-11, 8.29351033e-14, 3.17776472e-14, 1.16880646e-11,\n",
       "         1.12307431e-15, 1.77766573e-17, 1.29044824e-15, 4.61388524e-18,\n",
       "         4.45725565e-23, 2.90254472e-19, 3.95832910e-19, 5.83559696e-16,\n",
       "         2.19839016e-19, 1.29416258e-19, 8.66612571e-19, 4.24730607e-12,\n",
       "         7.73245877e-12, 2.29069801e-13, 3.72978287e-13, 8.63188806e-13,\n",
       "         4.57696455e-19, 1.06941470e-16, 1.95881350e-14, 2.03874329e-15,\n",
       "         2.67821209e-15, 2.38709046e-15, 1.84693708e-16, 5.91995011e-15,\n",
       "         5.36241494e-12, 2.40308591e-17, 6.67960058e-17, 8.11551076e-14],\n",
       "        [1.00000000e+00, 2.66287404e-12, 1.59619637e-17, 3.02760651e-16,\n",
       "         2.15594799e-11, 8.60986629e-14, 2.79615317e-14, 1.65680733e-11,\n",
       "         1.11391376e-15, 1.17755438e-17, 1.12870920e-15, 6.60543052e-18,\n",
       "         4.52516837e-23, 2.10998135e-19, 4.80803074e-19, 7.35334130e-16,\n",
       "         3.33258584e-19, 1.13428783e-19, 8.32968137e-19, 6.70169475e-12,\n",
       "         6.32461748e-12, 1.59381758e-13, 4.77193046e-13, 9.73687277e-13,\n",
       "         4.20041126e-19, 1.27785812e-16, 1.54506314e-14, 1.57836035e-15,\n",
       "         1.97145521e-15, 1.96738331e-15, 1.85668550e-16, 4.81020524e-15,\n",
       "         6.58784832e-12, 1.99292757e-17, 7.29077124e-17, 7.12679766e-14],\n",
       "        [1.00000000e+00, 3.25346802e-12, 1.21611167e-17, 2.57863775e-16,\n",
       "         1.78003948e-11, 8.73005281e-14, 2.49793625e-14, 2.20404597e-11,\n",
       "         1.15423337e-15, 8.70781716e-18, 1.05714169e-15, 9.14010258e-18,\n",
       "         4.72136060e-23, 1.68323810e-19, 5.67192628e-19, 9.16647409e-16,\n",
       "         4.69500478e-19, 1.06594707e-19, 8.45079095e-19, 9.94700329e-12,\n",
       "         5.26303133e-12, 1.20455769e-13, 5.88361825e-13, 1.10607346e-12,\n",
       "         3.98811148e-19, 1.50760443e-16, 1.28586166e-14, 1.32091432e-15,\n",
       "         1.57466178e-15, 1.71295420e-15, 1.87765473e-16, 4.19784065e-15,\n",
       "         7.86157164e-12, 1.71954521e-17, 8.05289642e-17, 6.51655786e-14],\n",
       "        [1.00000000e+00, 3.91801435e-12, 9.81944947e-18, 2.32635481e-16,\n",
       "         1.52831792e-11, 8.76385349e-14, 2.29441778e-14, 2.80066751e-11,\n",
       "         1.24074212e-15, 7.03662281e-18, 1.04790702e-15, 1.23758313e-17,\n",
       "         5.11194583e-23, 1.45289195e-19, 6.65626914e-19, 1.12972575e-15,\n",
       "         6.33036463e-19, 1.05556382e-19, 8.92193390e-19, 1.40219338e-11,\n",
       "         4.50183770e-12, 9.75597776e-14, 7.11448261e-13, 1.26958860e-12,\n",
       "         3.88474570e-19, 1.78764265e-16, 1.12352051e-14, 1.17071166e-15,\n",
       "         1.35305932e-15, 1.57218294e-15, 1.91579834e-16, 3.86329525e-15,\n",
       "         9.18325832e-12, 1.55559669e-17, 9.09964716e-17, 6.20789227e-14],\n",
       "        [1.00000000e+00, 4.69994963e-12, 8.42684541e-18, 2.20866528e-16,\n",
       "         1.36921256e-11, 8.73508283e-14, 2.19150056e-14, 3.41850194e-11,\n",
       "         1.38913065e-15, 6.10664805e-18, 1.09540641e-15, 1.63195621e-17,\n",
       "         5.78735939e-23, 1.34811053e-19, 7.90928130e-19, 1.37407326e-15,\n",
       "         8.34966036e-19, 1.10116493e-19, 9.75061378e-19, 1.89424223e-11,\n",
       "         3.98841507e-12, 8.48927862e-14, 8.66129434e-13, 1.47789235e-12,\n",
       "         3.78790017e-19, 2.16492172e-16, 1.02445323e-14, 1.08954293e-15,\n",
       "         1.25148504e-15, 1.52732121e-15, 1.97218374e-16, 3.67238842e-15,\n",
       "         1.04497123e-11, 1.48410595e-17, 1.05081670e-16, 6.21374425e-14],\n",
       "        [1.00000000e+00, 5.67575587e-12, 7.76868344e-18, 2.13993610e-16,\n",
       "         1.27077628e-11, 8.41260113e-14, 2.19046820e-14, 3.96999655e-11,\n",
       "         1.60409077e-15, 5.51820210e-18, 1.19404329e-15, 1.98683224e-17,\n",
       "         6.74860533e-23, 1.32047326e-19, 9.59677266e-19, 1.60476395e-15,\n",
       "         1.07866068e-18, 1.20191023e-19, 1.10424466e-18, 2.42657890e-11,\n",
       "         3.61442473e-12, 7.99290782e-14, 1.08821741e-12, 1.71205845e-12,\n",
       "         3.49718625e-19, 2.69172009e-16, 9.55252418e-15, 1.04314098e-15,\n",
       "         1.22676428e-15, 1.55831859e-15, 2.03635628e-16, 3.46006458e-15,\n",
       "         1.13861472e-11, 1.46880327e-17, 1.19273608e-16, 6.50096500e-14],\n",
       "        [1.00000000e+00, 6.88881330e-12, 7.66445124e-18, 1.93863276e-16,\n",
       "         1.16577407e-11, 7.31476850e-14, 2.22995518e-14, 4.32086381e-11,\n",
       "         1.75563725e-15, 4.88904233e-18, 1.29968143e-15, 1.99032923e-17,\n",
       "         7.52611484e-23, 1.27761251e-19, 1.15912064e-18, 1.64940693e-15,\n",
       "         1.28906369e-18, 1.32193479e-19, 1.28889649e-18, 2.79698955e-11,\n",
       "         3.15992588e-12, 7.94401369e-14, 1.39212470e-12, 1.82728377e-12,\n",
       "         2.84990294e-19, 3.26848203e-16, 8.47692871e-15, 9.52461126e-16,\n",
       "         1.16147709e-15, 1.57990248e-15, 2.07859609e-16, 3.02767713e-15,\n",
       "         1.15333125e-11, 1.37608600e-17, 1.18253807e-16, 6.69498162e-14],\n",
       "        [1.00000000e+00, 8.10895968e-12, 7.54545221e-18, 1.51044239e-16,\n",
       "         9.32045760e-12, 5.47458095e-14, 2.06100277e-14, 4.57198342e-11,\n",
       "         1.48963185e-15, 4.03007854e-18, 1.30840195e-15, 1.64353277e-17,\n",
       "         7.20957652e-23, 1.05445314e-19, 1.28527303e-18, 1.36500905e-15,\n",
       "         1.25544221e-18, 1.40859397e-19, 1.50780405e-18, 2.67935674e-11,\n",
       "         2.46404329e-12, 7.33132020e-14, 1.57131599e-12, 1.59531481e-12,\n",
       "         2.18682256e-19, 3.22521903e-16, 6.15502209e-15, 7.15745434e-16,\n",
       "         8.66765956e-16, 1.39256875e-15, 2.10743465e-16, 2.53338851e-15,\n",
       "         1.06495559e-11, 1.01455398e-17, 8.64668504e-17, 6.09299056e-14],\n",
       "        [1.00000000e+00, 9.05137510e-12, 6.03297149e-18, 1.33504198e-16,\n",
       "         5.80336387e-12, 4.24132741e-14, 1.41808401e-14, 5.44802675e-11,\n",
       "         9.52515548e-16, 3.65025953e-18, 1.23182879e-15, 2.29683141e-17,\n",
       "         7.01583291e-23, 7.06922446e-20, 1.23638397e-18, 1.20040048e-15,\n",
       "         9.60435273e-19, 1.77063206e-19, 1.84529436e-18, 2.19666386e-11,\n",
       "         1.91762686e-12, 4.96556057e-14, 1.11768440e-12, 1.28392696e-12,\n",
       "         2.71765408e-19, 2.06403400e-16, 3.53876813e-15, 4.39928028e-16,\n",
       "         5.22614829e-16, 9.76026748e-16, 2.30512711e-16, 3.30201967e-15,\n",
       "         9.48771357e-12, 5.98387625e-18, 5.37202423e-17, 5.26280476e-14],\n",
       "        [1.00000000e+00, 9.67037821e-12, 2.95648744e-18, 3.58430621e-16,\n",
       "         4.36756881e-12, 5.15385125e-14, 7.53623001e-15, 8.61863705e-11,\n",
       "         1.14380163e-15, 6.94380157e-18, 1.55664295e-15, 2.38500470e-16,\n",
       "         1.57843305e-22, 7.17849437e-20, 1.22873875e-18, 2.65194200e-15,\n",
       "         7.35605153e-19, 4.79581225e-19, 3.25763660e-18, 2.19380000e-11,\n",
       "         2.53942028e-12, 2.82446541e-14, 4.60600091e-13, 2.09917890e-12,\n",
       "         1.16725390e-18, 1.16489755e-16, 3.11093115e-15, 3.92489471e-16,\n",
       "         8.20778420e-16, 8.59246898e-16, 3.27283631e-16, 1.97359693e-14,\n",
       "         8.28752518e-12, 6.58527791e-18, 6.04908248e-17, 6.84161454e-14],\n",
       "        [1.00000000e+00, 2.40653799e-11, 4.96834299e-19, 5.01772238e-15,\n",
       "         9.66125270e-12, 8.38607138e-14, 3.66529283e-15, 1.24015381e-10,\n",
       "         1.01370828e-14, 9.60993356e-17, 4.44952718e-15, 7.23589161e-15,\n",
       "         4.88528508e-22, 7.18123725e-19, 2.12732656e-18, 1.58192957e-14,\n",
       "         6.04229588e-19, 3.44050101e-18, 2.52877104e-17, 2.17226480e-11,\n",
       "         1.18598360e-11, 2.83261201e-14, 3.22636855e-13, 6.67733794e-12,\n",
       "         5.52334014e-18, 1.77227695e-16, 1.21398617e-14, 1.11506583e-15,\n",
       "         2.59229555e-14, 3.21730727e-15, 8.38823822e-16, 3.33520321e-13,\n",
       "         1.20309865e-11, 2.68036578e-17, 1.81819897e-16, 7.63343449e-14]]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pre_slot_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
